#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import json
import time
import random
import pymysql
import logging
import hashlib
from datetime import datetime, timedelta

# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# 数据库配置
DB_CONFIG = {
    'host': 'localhost',
    'port': 3306,
    'user': 'root',
    'password': '13177936413qq',
    'db': 'codoon_data',
    'charset': 'utf8mb4'
}

class CodoonMockCrawler:
    """咕咚运动模拟爬虫类"""
    
    def __init__(self):
        """初始化爬虫"""
        self.conn = None
        self.cursor = None
        
        # 爬虫配置
        self.max_retries = 3
        
        # 用于生成更真实数据的预设值
        self.realistic_user_names = [
            "跑步达人", "马拉松爱好者", "晨跑族", "夜跑狂人", "越野跑者", 
            "健身达人", "运动博主", "铁人三项", "骑行达人", "健康生活家",
            "山地车手", "徒步爱好者", "运动教练", "瑜伽达人", "游泳健将",
            "舞蹈爱好者", "篮球达人", "足球爱好者", "网球高手", "羽毛球爱好者"
        ]
        
        self.realistic_locations = [
            "北京市朝阳区", "上海市浦东新区", "广州市天河区", "深圳市南山区", "杭州市西湖区",
            "成都市锦江区", "重庆市渝中区", "南京市鼓楼区", "武汉市武昌区", "西安市碑林区",
            "苏州市姑苏区", "厦门市思明区", "青岛市市南区", "大连市中山区", "长沙市岳麓区",
            "天津市和平区", "沈阳市沈河区", "济南市历下区", "郑州市金水区", "福州市鼓楼区"
        ]
        
        self.realistic_activities = [
            {
                "type": "run",
                "title_templates": [
                    "跑步{}公里，用时{}分钟",
                    "晨跑{}公里，感觉不错",
                    "夜跑{}公里，放松心情",
                    "午间慢跑{}公里",
                    "周末长跑{}公里，挑战自我"
                ],
                "description_templates": [
                    "今天天气很好，心情愉快，跑步节奏稳定。",
                    "今天有点累，但坚持下来了，继续保持！",
                    "尝试了新路线，风景很不错，值得推荐。",
                    "今天配速提高了，感觉进步不少。",
                    "和跑友一起跑步，互相鼓励，坚持到了终点。"
                ],
                "distance_range": (2.0, 15.0),
                "duration_per_km": (4.5, 7.0),  # 分钟/公里
                "calorie_per_km": (60, 80)  # 卡路里/公里
            },
            {
                "type": "bike",
                "title_templates": [
                    "骑行{}公里，用时{}分钟",
                    "周末骑行{}公里，探索新路线",
                    "公路骑行{}公里，速度提升",
                    "山地骑行{}公里，挑战自我",
                    "通勤骑行{}公里，低碳出行"
                ],
                "description_templates": [
                    "今天骑行路线很平坦，速度很快，心情愉悦。",
                    "爬坡骑行，很有挑战性，但风景绝佳。",
                    "和车友一起骑行，互相鼓励，完成了长距离挑战。",
                    "今天尝试了新的骑行姿势，感觉效率提升了。",
                    "雨后骑行，空气清新，路面有点湿滑，需要小心。"
                ],
                "distance_range": (10.0, 60.0),
                "duration_per_km": (2.0, 4.0),  # 分钟/公里
                "calorie_per_km": (25, 40)  # 卡路里/公里
            },
            {
                "type": "walk",
                "title_templates": [
                    "健走{}公里，锻炼身体",
                    "午休散步{}公里，放松心情",
                    "傍晚漫步{}公里，欣赏美景",
                    "快走{}公里，燃脂减肥",
                    "户外健走{}公里，呼吸新鲜空气"
                ],
                "description_templates": [
                    "每天坚持走路，让身体更健康。",
                    "边走边欣赏沿途风景，心情放松。",
                    "快走锻炼，感觉很有活力。",
                    "雨后漫步，空气清新，很舒适。",
                    "和朋友一起健走，边走边聊，很愉快。"
                ],
                "distance_range": (1.0, 8.0),
                "duration_per_km": (10.0, 15.0),  # 分钟/公里
                "calorie_per_km": (40, 60)  # 卡路里/公里
            },
            {
                "type": "swim",
                "title_templates": [
                    "游泳{}米，畅快淋漓",
                    "自由泳{}米，提升技巧",
                    "蛙泳训练{}米，持续进步",
                    "泳池锻炼{}米，保持状态",
                    "游泳{}米，放松身心"
                ],
                "description_templates": [
                    "今天游泳感觉很好，动作更协调了。",
                    "尝试了新的呼吸节奏，感觉很有效。",
                    "坚持每周游泳，身体素质明显提升。",
                    "游泳后全身放松，睡眠质量也提高了。",
                    "和教练一起训练，纠正了一些动作问题。"
                ],
                "distance_range": (500, 3000),
                "duration_per_100m": (1.5, 3.0),  # 分钟/100米
                "calorie_per_100m": (10, 15)  # 卡路里/100米
            }
        ]
        
        self.realistic_routes = [
            {
                "title_templates": [
                    "{}公园跑步路线",
                    "{}河畔慢跑道",
                    "{}周边5公里经典跑道",
                    "{}健身步道",
                    "{}山间越野跑道"
                ],
                "description_templates": [
                    "这是一条非常适合晨跑的路线，路面平整，人流量小，空气清新。",
                    "这条路线沿河而建，风景优美，跑步的同时可以欣赏河景。",
                    "这是一条具有一定坡度的路线，适合想要提升体能的跑友。",
                    "这条路线环绕公园一周，全程有树荫，夏天跑步也不会太晒。",
                    "这是一条结合了道路和越野的混合路线，有一定的挑战性。"
                ],
                "distance_range": (3.0, 10.0),
                "location_landmarks": [
                    "奥林匹克公园", "世纪公园", "滨江公园", "莲花山公园", "紫竹公园",
                    "珠江", "钱塘江", "松花江", "汉江", "海河",
                    "健身步道", "城市绿道", "森林公园", "湿地公园", "山地公园"
                ]
            },
            {
                "title_templates": [
                    "{}自行车专用道",
                    "{}环线骑行路线",
                    "{}至{}经典骑行路线",
                    "{}山地车道",
                    "{}风景骑行路线"
                ],
                "description_templates": [
                    "这是一条平坦的骑行专用道，适合休闲骑行和新手练习。",
                    "这条环线路线风景优美，骑行过程中可以欣赏到不同的景色。",
                    "这是一条城市间的骑行路线，沿途有多个休息点和补给站。",
                    "这条山地车道有一定的技术难度，适合有经验的骑行者挑战。",
                    "这是一条风景优美的骑行路线，非常适合周末骑行和拍照。"
                ],
                "distance_range": (15.0, 80.0),
                "location_landmarks": [
                    "自行车专用道", "绿道", "滨海公路", "山间公路", "环湖公路",
                    "城市绿道", "森林公园", "风景区", "乡村小道", "城市快速路"
                ]
            }
        ]
        
    def init_database(self):
        """初始化数据库连接和表结构"""
        try:
            self.conn = pymysql.connect(**DB_CONFIG)
            self.cursor = self.conn.cursor()
            
            # 创建表结构
            self._create_tables()
            
            logger.info("数据库初始化成功")
            return True
        except Exception as e:
            logger.error(f"数据库初始化失败: {e}")
            if self.conn:
                self.conn.rollback()
            return False
    
    def _create_tables(self):
        """创建数据表"""
        # 用户表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `users` (
            `user_id` VARCHAR(50) PRIMARY KEY,
            `nickname` VARCHAR(100),
            `avatar_url` VARCHAR(255),
            `gender` VARCHAR(10),
            `location` VARCHAR(100),
            `follower_count` INT,
            `following_count` INT,
            `post_count` INT,
            `created_at` DATETIME
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        # 运动记录表 - 增加route_data字段长度
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `activities` (
            `activity_id` VARCHAR(50) PRIMARY KEY,
            `user_id` VARCHAR(50),
            `type` VARCHAR(50),
            `title` VARCHAR(255),
            `description` TEXT,
            `distance` FLOAT,
            `duration` INT,
            `calorie` FLOAT,
            `start_time` DATETIME,
            `likes_count` INT,
            `comments_count` INT,
            `route_data` MEDIUMTEXT,
            `created_at` DATETIME,
            FOREIGN KEY (`user_id`) REFERENCES `users`(`user_id`) ON DELETE CASCADE
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        # 运动路线表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `routes` (
            `route_id` VARCHAR(50) PRIMARY KEY,
            `title` VARCHAR(255),
            `description` TEXT,
            `distance` FLOAT,
            `author_id` VARCHAR(50),
            `likes_count` INT,
            `used_count` INT,
            `location` VARCHAR(255),
            `created_at` DATETIME,
            FOREIGN KEY (`author_id`) REFERENCES `users`(`user_id`) ON DELETE CASCADE
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        self.conn.commit()
    
    def _generate_mock_activities(self, size=20):
        """生成模拟活动数据用于测试"""
        mock_activities = []
        for i in range(size):
            activity_id = f"activity_{int(time.time())}_{random.randint(1000, 9999)}"
            user_id = f"user_{random.randint(1000, 9999)}"
            
            # 随机选择活动类型
            activity_type_data = random.choice(self.realistic_activities)
            activity_type = activity_type_data["type"]
            
            # 生成合理的活动距离
            if activity_type == "swim":
                distance = random.randint(*activity_type_data["distance_range"]) / 1000  # 转换为公里
                duration_per_unit = random.uniform(*activity_type_data["duration_per_100m"]) 
                duration = int(distance * 1000 / 100 * duration_per_unit * 60)  # 转换为秒
                calorie_per_unit = random.uniform(*activity_type_data["calorie_per_100m"])
                calorie = distance * 1000 / 100 * calorie_per_unit
            else:
                distance = round(random.uniform(*activity_type_data["distance_range"]), 2)
                duration_per_unit = random.uniform(*activity_type_data["duration_per_km"])
                duration = int(distance * duration_per_unit * 60)  # 转换为秒
                calorie_per_unit = random.uniform(*activity_type_data["calorie_per_km"])
                calorie = distance * calorie_per_unit
            
            # 生成活动标题
            title_template = random.choice(activity_type_data["title_templates"])
            if activity_type == "swim":
                title = title_template.format(int(distance * 1000))
            else:
                title = title_template.format(distance, int(duration/60))
            
            # 生成活动描述
            description = random.choice(activity_type_data["description_templates"])
            
            # 生成合理的时间
            current_time = datetime.now()
            days_ago = random.randint(1, 30)
            hours = random.randint(6, 21)  # 合理的运动时间
            start_time = current_time - timedelta(days=days_ago)
            start_time = start_time.replace(hour=hours, minute=random.randint(0, 59))
            
            # 生成点赞和评论数
            likes_count = int(random.paretovariate(1.5)) % 100  # 使用Pareto分布更真实
            comments_count = int(likes_count * random.uniform(0.1, 0.5))
            
            # 构造活动数据
            user_name_prefix = random.choice(self.realistic_user_names)
            user_name = f"{user_name_prefix}{random.randint(10, 999)}"
            
            # 构造路线数据（简化版，减少点数量）
            route_data = {
                "start_point": {
                    "lat": 39.90 + random.uniform(-0.1, 0.1),
                    "lng": 116.40 + random.uniform(-0.1, 0.1)
                },
                "end_point": {
                    "lat": 39.90 + random.uniform(-0.1, 0.1),
                    "lng": 116.40 + random.uniform(-0.1, 0.1)
                },
                "path_points": []
            }
            
            # 生成路径点 - 限制点数量避免数据过大
            num_points = min(int(distance * 5), 50)  # 每公里5个点，最多50个点
            for j in range(num_points):
                route_data["path_points"].append({
                    "lat": route_data["start_point"]["lat"] + (route_data["end_point"]["lat"] - route_data["start_point"]["lat"]) * j / num_points + random.uniform(-0.001, 0.001),
                    "lng": route_data["start_point"]["lng"] + (route_data["end_point"]["lng"] - route_data["start_point"]["lng"]) * j / num_points + random.uniform(-0.001, 0.001),
                    "timestamp": int(start_time.timestamp()) + int(duration * j / num_points),
                    "altitude": 50 + random.uniform(-10, 20),
                    "speed": random.uniform(2.0, 5.0) if activity_type == "run" else (
                              random.uniform(15.0, 30.0) if activity_type == "bike" else
                              random.uniform(1.0, 2.0))
                })
            
            mock_activity = {
                'id': activity_id,
                'type': activity_type,
                'title': title,
                'description': description,
                'distance': distance,
                'duration': duration,
                'calorie': round(calorie, 2),
                'start_time': int(start_time.timestamp()),
                'likes_count': likes_count,
                'comments_count': comments_count,
                'route_data': route_data,
                'user': {
                    'id': user_id,
                    'nickname': user_name,
                    'avatar': f"https://api.codoon.com/avatar/{user_id}.jpg",
                    'gender': random.choice(['male', 'female']),
                    'location': random.choice(self.realistic_locations)
                }
            }
            mock_activities.append(mock_activity)
        
        return mock_activities
    
    def _generate_mock_routes(self, size=20):
        """生成模拟路线数据用于测试"""
        mock_routes = []
        for i in range(size):
            route_id = f"route_{int(time.time())}_{random.randint(1000, 9999)}"
            author_id = f"user_{random.randint(1000, 9999)}"
            
            # 随机选择路线类型
            route_type_data = random.choice(self.realistic_routes)
            
            # 生成合理的路线距离
            distance = round(random.uniform(*route_type_data["distance_range"]), 2)
            
            # 生成路线标题
            location_landmark = random.choice(route_type_data["location_landmarks"])
            title_template = random.choice(route_type_data["title_templates"])
            
            # 处理不同的标题模板情况
            if "{}至{}" in title_template:
                location_landmark2 = random.choice(route_type_data["location_landmarks"])
                while location_landmark2 == location_landmark:
                    location_landmark2 = random.choice(route_type_data["location_landmarks"])
                title = title_template.format(location_landmark, location_landmark2)
            else:
                title = title_template.format(location_landmark)
            
            # 生成路线描述
            description = random.choice(route_type_data["description_templates"])
            
            # 生成合理的点赞和使用数
            likes_count = int(random.paretovariate(1.2)) % 200  # 使用Pareto分布更真实
            used_count = int(likes_count * random.uniform(0.5, 3.0))
            
            # 随机选择地点
            location = random.choice(self.realistic_locations)
            
            # 生成作者名称
            user_name_prefix = random.choice(self.realistic_user_names)
            user_name = f"{user_name_prefix}{random.randint(10, 999)}"
            
            mock_route = {
                'id': route_id,
                'title': title,
                'description': description,
                'distance': distance,
                'likes_count': likes_count,
                'used_count': used_count,
                'location': location,
                'author': {
                    'id': author_id,
                    'nickname': user_name,
                }
            }
            mock_routes.append(mock_route)
        
        return mock_routes
    
    def _generate_mock_user(self, user_id):
        """生成模拟用户数据用于测试"""
        # 提取用户ID中的数字部分
        id_num = int(user_id.split('_')[-1]) if '_' in user_id else random.randint(1000, 9999)
        
        # 生成更真实的昵称
        user_name_prefix = random.choice(self.realistic_user_names)
        nickname = f"{user_name_prefix}{id_num % 1000}"
        
        # 根据用户ID确定性地生成一些属性，使同一用户的数据保持一致
        random.seed(id_num)
        gender = random.choice(['male', 'female'])
        location = random.choice(self.realistic_locations)
        
        # 生成合理的社交数据
        follower_base = int(random.paretovariate(1.5)) % 500  # 基础粉丝数
        followers = max(follower_base, 5)  # 至少有5个粉丝
        following = int(followers * random.uniform(0.5, 1.5))  # 关注数与粉丝数相关
        post_count = int(followers * random.uniform(0.2, 1.0))  # 发帖数与粉丝数相关
        
        # 重置随机种子
        random.seed()
        
        return {
            'id': user_id,
            'nickname': nickname,
            'avatar': f"https://api.codoon.com/avatar/{user_id}.jpg",
            'gender': gender,
            'location': location,
            'follower_count': followers,
            'following_count': following,
            'post_count': post_count
        }
    
    def save_user(self, user_data):
        """保存用户数据"""
        try:
            if not user_data or not user_data.get('id'):
                return False
            
            sql = """
            INSERT INTO `users` 
            (`user_id`, `nickname`, `avatar_url`, `gender`, `location`,
             `follower_count`, `following_count`, `post_count`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `nickname`=%s, `avatar_url`=%s, `gender`=%s, `location`=%s,
            `follower_count`=%s, `following_count`=%s, `post_count`=%s
            """
            
            now = datetime.now()
            values = (
                user_data.get('id', ''),
                user_data.get('nickname', ''),
                user_data.get('avatar', ''),
                user_data.get('gender', ''),
                user_data.get('location', ''),
                user_data.get('follower_count', 0),
                user_data.get('following_count', 0),
                user_data.get('post_count', 0),
                now,
                # 更新值
                user_data.get('nickname', ''),
                user_data.get('avatar', ''),
                user_data.get('gender', ''),
                user_data.get('location', ''),
                user_data.get('follower_count', 0),
                user_data.get('following_count', 0),
                user_data.get('post_count', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存用户数据成功: {user_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存用户数据失败: {e}")
            self.conn.rollback()
            return False
    
    def save_activity(self, activity_data):
        """保存运动记录"""
        try:
            if not activity_data or not activity_data.get('id'):
                return False
            
            # 确保用户存在
            user_id = activity_data.get('user', {}).get('id')
            if user_id:
                self.cursor.execute("SELECT 1 FROM `users` WHERE `user_id` = %s", (user_id,))
                if not self.cursor.fetchone():
                    self.cursor.execute("""
                    INSERT INTO `users` (`user_id`, `nickname`, `created_at`)
                    VALUES (%s, %s, %s)
                    """, (user_id, activity_data.get('user', {}).get('nickname', ''), datetime.now()))
                    self.conn.commit()
            
            sql = """
            INSERT INTO `activities` 
            (`activity_id`, `user_id`, `type`, `title`, `description`, `distance`,
             `duration`, `calorie`, `start_time`, `likes_count`, `comments_count`,
             `route_data`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `likes_count`=%s, `comments_count`=%s
            """
            
            now = datetime.now()
            
            # 处理时间戳
            try:
                start_time = datetime.fromtimestamp(activity_data.get('start_time', 0))
            except:
                start_time = now
            
            # 处理JSON数据
            try:
                route_data = json.dumps(activity_data.get('route_data', {}))
            except:
                route_data = '{}'
            
            values = (
                activity_data.get('id', ''),
                user_id,
                activity_data.get('type', ''),
                activity_data.get('title', ''),
                activity_data.get('description', ''),
                activity_data.get('distance', 0),
                activity_data.get('duration', 0),
                activity_data.get('calorie', 0),
                start_time,
                activity_data.get('likes_count', 0),
                activity_data.get('comments_count', 0),
                route_data,
                now,
                # 更新值
                activity_data.get('likes_count', 0),
                activity_data.get('comments_count', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存运动记录成功: {activity_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存运动记录失败: {e}")
            self.conn.rollback()
            return False
    
    def save_route(self, route_data):
        """保存路线数据"""
        try:
            if not route_data or not route_data.get('id'):
                return False
            
            sql = """
            INSERT INTO `routes` 
            (`route_id`, `title`, `description`, `distance`, `author_id`,
             `likes_count`, `used_count`, `location`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `title`=%s, `description`=%s, `likes_count`=%s, `used_count`=%s
            """
            
            now = datetime.now()
            values = (
                route_data.get('id', ''),
                route_data.get('title', ''),
                route_data.get('description', ''),
                route_data.get('distance', 0),
                route_data.get('author', {}).get('id', ''),
                route_data.get('likes_count', 0),
                route_data.get('used_count', 0),
                route_data.get('location', ''),
                now,
                # 更新值
                route_data.get('title', ''),
                route_data.get('description', ''),
                route_data.get('likes_count', 0),
                route_data.get('used_count', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存路线数据成功: {route_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存路线数据失败: {e}")
            self.conn.rollback()
            return False
    
    def random_sleep(self, min_sleep=1, max_sleep=3):
        """随机休眠，避免请求过快"""
        sleep_time = random.uniform(min_sleep, max_sleep)
        time.sleep(sleep_time)
    
    def run(self, max_pages=3):
        """运行爬虫程序，使用模拟数据"""
        if not self.init_database():
            logger.error("数据库初始化失败，爬虫程序终止")
            return
        
        try:
            # 爬取热门动态
            logger.info("开始生成模拟运动动态...")
            for page in range(1, max_pages + 1):
                activities = self._generate_mock_activities(size=20)
                logger.info(f"第{page}页，生成了{len(activities)}条模拟动态")
                
                for activity in activities:
                    # 保存用户信息
                    user_data = activity.get('user')
                    if user_data and user_data.get('id'):
                        user_detail = self._generate_mock_user(user_data.get('id'))
                        if user_detail:
                            self.save_user(user_detail)
                    
                    # 保存活动记录
                    self.save_activity(activity)
                    self.random_sleep(0.1, 0.5)  # 减少等待时间，因为是模拟数据
                
                self.random_sleep(0.5, 1)
            
            # 爬取热门路线
            logger.info("开始生成模拟路线...")
            for page in range(1, max_pages + 1):
                routes = self._generate_mock_routes(size=20)
                logger.info(f"第{page}页，生成了{len(routes)}条模拟路线")
                
                for route in routes:
                    # 保存用户信息
                    author_data = route.get('author')
                    if author_data and author_data.get('id'):
                        user_detail = self._generate_mock_user(author_data.get('id'))
                        if user_detail:
                            self.save_user(user_detail)
                    
                    # 保存路线数据
                    self.save_route(route)
                    self.random_sleep(0.1, 0.5)
                
                self.random_sleep(0.5, 1)
            
            logger.info("模拟爬虫程序执行完成!")
        
        except Exception as e:
            logger.error(f"模拟爬虫程序执行过程中出错: {e}")
        
        finally:
            # 关闭数据库连接
            if self.cursor:
                self.cursor.close()
            if self.conn:
                self.conn.close()

if __name__ == "__main__":
    crawler = CodoonMockCrawler()
    crawler.run(max_pages=3)  # 爬取3页模拟数据 